# -*- coding: utf-8 -*-
from datetime import timedelta
from jms.dim import jms_dim__dim_lmdm_sys_network
from utils.operators.cluster_for_spark_sql_operator  import SparkSqlOperator
from jms.dwd.oms.dwd_yl_oms_oms_waybill_incre_dt import jms_dwd__dwd_yl_oms_oms_waybill_incre_dt

from jms.dwd.tms.dwd_tmsnew_shipment_stop_union_base_dt import jms_dwd__dwd_tmsnew_shipment_stop_union_base_dt
from jms.dwd.tab.dwd_tab_barscan_loading_base_dt import jms_dwd__dwd_tab_barscan_loading_base_dt
from jms.dwd.tab.dwd_tab_barscan_unloading_base_dt import jms_dwd__dwd_tab_barscan_unloading_base_dt
from jms.dwd.tab.dwd_tab_barscan_arrival_base_dt import jms_dwd__dwd_tab_barscan_arrival_base_dt
from jms.dwd.tab.dwd_tab_barscan_send_base_dt import jms_dwd__dwd_tab_barscan_send_base_dt
from jms.dwd.tab.dwd_tab_barscan_bagging_base_dt import jms_dwd__dwd_tab_barscan_bagging_base_dt
from jms.dwd.tab.dwd_tab_barscan_other_base_dt import jms_dwd__dwd_tab_barscan_other_base_dt
from jms.dim.dim_lmdm_sys_network_expand import jms_dim__dim_lmdm_sys_network_expand
from jms.dwd.tab.dwd_tab_reback_transfer_express_base_dt import jms_dwd__dwd_tab_reback_transfer_express_base_dt
from jms.dim.dim_sys_manage_region import jms_dim__dim_sys_manage_region

jms_dm__dm_cn_breakoff_detail_total_dt = SparkSqlOperator(
    task_id='jms_dm__dm_cn_breakoff_detail_total_dt',
    email=['jarl.huang@jtexpress.com','yl_bigdata@yl-scm.com'],
    name='jms_dm__dm_cn_breakoff_detail_total_dt_{{ execution_date | date_add(-1) | cst_ds }}',
    sql='jms/dm/dm_cn_breakoff_detail_total_dt/execute.hql',
    pool_slots=4,
    executor_cores=8,
    driver_cores=6,
    executor_memory='12G',
    driver_memory='8G',
    num_executors=70,
    # num_executors=10,  # spark.dynamicAllocation.enabled 为 True 时，num_executors 表示最少 Executor 数
    conf={'spark.dynamicAllocation.maxExecutors': 100,
          'spark.sql.sources.partitionOverwriteMode': 'dynamic',
          'spark.dynamicallocation.enabled': 'true',
          'spark.dynamicAllocation.cachedExecutorIdleTimeout': 120,
          'spark.shuffle.service.enabled': 'true',
          'spark.executor.memoryOverhead': '2G',  # 堆外内存
          'spark.sql.shuffle.partitions': 2000
          },
    hiveconf={'hive.exec.dynamic.partition': 'true',
              'hive.exec.dynamic.partition.mode': 'nonstrict',
              'hive.exec.max.dynamic.partitions.pernode': 5000,
              'hive.exec.max.dynamic.partitions': 30000
              },
    yarn_queue='pro',
    execution_timeout=timedelta(minutes=120),
)

jms_dm__dm_cn_breakoff_detail_total_dt << [
                                           jms_dwd__dwd_yl_oms_oms_waybill_incre_dt,
                                            jms_dwd__dwd_tmsnew_shipment_stop_union_base_dt,
                                            jms_dwd__dwd_tab_barscan_loading_base_dt,
                                            jms_dwd__dwd_tab_barscan_unloading_base_dt,
                                            jms_dwd__dwd_tab_barscan_arrival_base_dt,
                                            jms_dwd__dwd_tab_barscan_send_base_dt,
                                            jms_dwd__dwd_tab_barscan_bagging_base_dt,
                                            jms_dwd__dwd_tab_barscan_other_base_dt,
                                            jms_dim__dim_lmdm_sys_network_expand,
                                           jms_dim__dim_lmdm_sys_network,
                                            jms_dwd__dwd_tab_reback_transfer_express_base_dt,
                                            jms_dim__dim_sys_manage_region]
